# -*- coding:utf-8 -*-
"""

"""
# -*- coding:utf-8 -*-
"""

"""
# -*- coding:utf-8 -*-
"""

"""

import requests
from bs4 import BeautifulSoup
import time
from WriteData import writedata
import json
import re


def get_html_text(url):
    headers = {
        'Accept': 'application/json, text/plain, */*',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Connection': 'keep-alive',
        'Cookie': '_ga=GA1.2.210448367.1539327655; _gid=GA1.2.466958709.1539327655; _ym_uid=1539327656731870621; _ym_d=1539327656; _ym_isad=2; _ym_visorc_46553805=w; XSRF-TOKEN=eyJpdiI6IlpUdlF3d3ZrVGxsaUJlbHJhNnlQZkE9PSIsInZhbHVlIjoiUDA0bGhMZHd1ZWl1cFJ0eTk4dWVQNTNCMjh4MWZicFJPRjBFWWM4SWVYbG1YZG9IeUk3VmthNkxxUjQ3MG5Kd2JWYXBIOHpYWXZTbm4rWCtEMldzZmc9PSIsIm1hYyI6IjdjNzlhYTkzNTEyMDQ2Y2E4ZWI2MzlkMDU4NjBmNzFhNzAzN2ZhNWI0MzQxMDNhNWUwZGZjMzc4YWY4ZTQ3ODYifQ%3D%3D; icoratingsession=eyJpdiI6IitpOUNuWkJXNG8xNmQ2SDNxamJyVHc9PSIsInZhbHVlIjoiS3cybmNvZll5OXRwYXdHR2Q1UENiYUlXcGRNTUtObkliSWJjdHN2VFBsM2IzK28rTll2SVVtckF4SUkrWHdUcjVlUW52K0FIMUJ0QklXakNPYzk1THc9PSIsIm1hYyI6ImIwYWMyYjNhNjY0YTMyMzEwYTc1N2YwNzhkMDQwODcxNzUwZjk0MjA2ZjJhZWVmZWI4N2M4ZGU5OThlOWViZWMifQ%3D%3D; locale=eyJpdiI6IlwvMnBRTjJkWDFFRzBGVmVIRXk5RktRPT0iLCJ2YWx1ZSI6InlFMEVDSWZKNytRcDhyMmY1YlVSaFE9PSIsIm1hYyI6IjgyOGRhMGUzY2M2NDk0OWE0ZDA5M2JkNzU3OGRiM2U1ODk4YjM5MjRlMjE0ZmUzNjMzYWQwY2M1ZmU5ZTcwODcifQ%3D%3D',
        'Host': 'icorating.com',
        'Referer': 'https://icorating.com/analytics/indepth/',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome / 69.0.3497.100 Safari / 537.36',
        'X-XSRF-TOKEN': 'eyJpdiI6IlpUdlF3d3ZrVGxsaUJlbHJhNnlQZkE9PSIsInZhbHVlIjoiUDA0bGhMZHd1ZWl1cFJ0eTk4dWVQNTNCMjh4MWZicFJPRjBFWWM4SWVYbG1YZG9IeUk3VmthNkxxUjQ3MG5Kd2JWYXBIOHpYWXZTbm4rWCtEMldzZmc9PSIsIm1hYyI6IjdjNzlhYTkzNTEyMDQ2Y2E4ZWI2MzlkMDU4NjBmNzFhNzAzN2ZhNWI0MzQxMDNhNWUwZGZjMzc4YWY4ZTQ3ODYifQ==', }
    try:
        r = requests.get(url, timeout=30, headers=headers)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except Exception as e:
        print(e)
        return None


def get_urls(url):
    text = get_html_text(url)
    if not text:
        return None
    text = json.loads(text)
    data = text['ratings']['data']
    if not data:
        return None
    urllist = [date['url'] for date in data]
    for url in urllist:
        with open('url.txt', 'a', encoding='utf-8') as f:
            f.write(url + '\n')
    # return (title, ctime, expiretime, urllist)


def a():
    a = 1
    while a < 21:
        url = 'https://icorating.com/analytics/indepth/json/?page={}'
        url = url.format(a)
        get_urls(url)
        print(url)
        print('----------------------')
        time.sleep(3)
        a += 1


def get_datas(url):
    text = get_html_text(url)
    print('{}页面获取中'.format(url))
    if not text:
        return None
    context = ''
    soup = BeautifulSoup(text, 'lxml')
    title = soup.select_one('h1').get_text()
    time_Rating = soup.select('.c-card div')[3].get_text()[-15:]
    regex = re.compile(r'<div id="title-\d+"><[\s\S]*?body>([\s\S]*?)</body>')
    datas = re.findall(regex, text)
    for x in datas:
        context += x
    d = {"title": title, "time_Rating": time_Rating, "context": context}
    writedata(d)


def result():
    a = 1
    while a < 21:
        b = 0
        url = 'https://portal.one.top/app/v1/information/queryInformationReport?page={}&size=5'
        url = url.format(a)
        data = get_urls(url)
        titlelist = data[0]
        ctimelist = data[1]
        urllist = data[2]
        print(a)
        # time.sleep(10)
        for url in urllist:
            title = titlelist[b]
            ctime = ctimelist[b]
            context = get_datas(url)
            d = {"title": title, "ctime": ctime, "context": context}
            writedata(d)
            b += 1
        a += 1


def a():
    with open('ICOurl.txt', encoding='utf-8') as f:
        for x in f:
            get_datas(x.strip())

a()
4

# l = [
#      "https://icorating.com/analytics/posticorating/icon-post-ico/",]
#
# for x in l:
#     get_datas(x.strip())
